In [1]:
%matplotlib notebook
from IPython.core.display import display, HTML
display(HTML("<style>.container { width:100% !important; }</style>"))
import os, sys
path_to_add = '/home/xpanda/projects/mnubo/smart_alerts_project/hsb_admodel_mnubo'
if path_to_add not in sys.path:
    sys.path.append(path_to_add)
In [3]:
import numpy as np
import pandas as pd
from custom_lib import ModelSaver
from anomaly_plot import gen_sankey_tsdata
from anomaly_plot import AnomalyPlotter
import plotnine as p9

mnubo_tenant = 'hsbprod'
ms = ModelSaver(mnubo_tenant)

1. Model deployment status

21 sensors are found in the hsbprod tenants and the following input parameter variations are used to create models.

1. channel : temp (univariate) or temp & hum (bivariate)
2. ts_len : 12 (3 hours), 24 (6 hours) and 96 (24 hours)

Total number of models deployed: 126 (88 have been trained)

Each model uses data of 2 weeks until 2020-04-14 to learn patterns and caculates anomaly values ranging from 0 (normal) to 1 (abnormal) thereafter.

The models are scheduled to run twice a day and computed anomaly values are saved as events in Mnubo Smartobject platform.

In [39]:
psets = ms.get_unique_model_parameter_sets()
acc_sensor_info = pd.read_csv(os.path.join('/home/xpanda/datadir/mnubo/smart_alerts_project/data', 'sensors_in_use_20200511.csv'))
acc_info = acc_sensor_info.drop(columns='node_id')\
    .rename(columns={'x_device_id':'node_id'})\
    [['Insured Name','node_id']].dropna()
acc_info['node_id'] = acc_info.node_id.astype(int).astype(str)
psets['node_id'] = psets.node_ids.apply(lambda x: x.split(',')[0])
pd.merge(psets, acc_info, how='left', on='node_id').sort_values(['ts_len','model_id']).reset_index(drop=True)
Out[39]:
x_device_id x_object_type model_id unique_id node_id node_ids mse_threshold_train nbr_days ts_len channel end_datetime_training heartbeat_interval status Insured Name
0 ada_parms_e9a97d37d8e74e7d758339df028a91fd7927f8fc_20200515000322306889Z model_lstm 0cd582b2cd1d18351f6df447e5f58c8518d4b4bd NaN 179172 179172 0.001 14 12 temp 2020-04-14 00:00:00+00:00 15T active Max Downtown
1 ada_parms_5d8f862b141264a3b31bc2b562b4273a5e6dfc66_20200514162010021360Z model_lstm 111867f9a2c01c8eebf82e47f6adba573531533e NaN 175515 175515,175515 0.001 14 12 temp,hum 2020-04-14 00:00:00+00:00 15T active EMC - Elkhorn
2 ada_parms_61e7c8bd316fc31cc1b1a0f13e0d875058e64bb9_20200514165532862223Z model_lstm 1cb8f915c47004231b92a0373a4f8bddfa7656f1 NaN 207300 207300 0.001 14 12 temp 2020-04-14 00:00:00+00:00 15T active Kane's Market
3 ada_parms_83f4bc949fdce85b230a875bea631eeba3b1f45e_20200514163554898513Z model_lstm 21925e1f9f42bc7b3595a5f44ba85741146dc797 NaN 207298 207298,207298 0.001 14 12 temp,hum 2020-04-14 00:00:00+00:00 15T active Kane's Market
4 ada_parms_d1a3a8be6eebf3b0426e5cc47ed9e0096f3e550e_20200514200133057001Z model_lstm 23d719d3f297e36fba169967bb071746171b15ab NaN 175509 175509,175509 0.001 14 12 temp,hum 2020-04-14 00:00:00+00:00 15T active EMC Guthrie Center
5 ada_parms_38ec9055c2e4264fb53e254d164dad9c754e33a2_20200514230125597088Z model_lstm 289109d32e13e67dd7ede66a64e371a6f8e2e712 NaN 179177 179177 0.001 14 12 temp 2020-04-14 00:00:00+00:00 15T active Max Downtown
6 ada_parms_a8ef54c0b15d93d71282f14d8d5da7aae999056a_20200515000749263292Z model_lstm 2908922a82fc1d718f5d0ba2b29602430fb3012c NaN 169743 169743 0.001 14 12 temp 2020-04-14 00:00:00+00:00 15T active Max's Oyster Bar
7 ada_parms_6fbb33ad3e0722b11a2333e2f31a7b9af302d7c1_20200514225604395572Z model_lstm 37a9de977ea713e6eb12a405652fb34c7eb78f78 NaN 207192 207192,207192 0.001 14 12 temp,hum 2020-04-14 00:00:00+00:00 15T active On20 Restaurant
8 ada_parms_706de91bae45df0b66e5c90366bc5403c8e145a6_20200514231538150938Z model_lstm 45d669324b941bd4abe5c29264408b4e338ebf71 NaN 207298 207298 0.001 14 12 temp 2020-04-14 00:00:00+00:00 15T active Kane's Market
9 ada_parms_7dbabcb44276694de20549fc0bca2c85aa7c6ad1_20200514180532144149Z model_lstm 4c71a5479c712e7fa4c6f5d440484ef09b9b8d43 NaN 195514 195514 0.001 14 12 temp 2020-04-14 00:00:00+00:00 15T active Pomperaug Woods
10 ada_parms_08d7389fd51990830f503eb61b2dc8417351ef69_20200514175058012304Z model_lstm 53d751b4e1b9a04e4ff4d296ff63df3d451f3a2e NaN 207294 207294,207294 0.001 14 12 temp,hum 2020-04-14 00:00:00+00:00 15T active Kane's Market
11 ada_parms_8be438c38a23cae220526b8cd736a1cfb14c5959_20200514221435201655Z model_lstm 55cefac593c5cff3e62fc8145b29ab25009ebfe8 NaN 207296 207296,207296 0.001 14 12 temp,hum 2020-04-14 00:00:00+00:00 15T active Kane's Market
12 ada_parms_d1a499cf5ab9ff0a521ac324703d520dd29c48e5_20200514161352626689Z model_lstm 5844673b32e054ffefdc87dd18a8050f5e485836 NaN 195517 195517 0.001 14 12 temp 2020-04-14 00:00:00+00:00 15T active Pomperaug Woods
13 ada_parms_2850cdd34c887d543e8c40872a82df4abd3e35e5_20200514164109516337Z model_lstm 59099743b1a578352549e0473ae63d623ff2dc62 NaN 207295 207295 0.001 14 12 temp 2020-04-14 00:00:00+00:00 15T active Kane's Market
14 ada_parms_e09315a6558f0b929f2db3b4af12027b2065e4ea_20200514192617004951Z model_lstm 60f34c74a6854e9334c22935be78c3c7ce1ba341 NaN 207303 207303 0.001 14 12 temp 2020-04-14 00:00:00+00:00 15T active Kane's Market
15 ada_parms_3c64934c80ef4c71bbb14a4ba80b40ed493b6c10_20200514205406996303Z model_lstm 67f7413e77f26b0b890f3a336c096e38df39c72b NaN 207301 207301,207301 0.001 14 12 temp,hum 2020-04-14 00:00:00+00:00 15T active Kane's Market
16 ada_parms_2dd8836a52d2653fb8c4099347ecf8552d54ebbf_20200514202314912773Z model_lstm 6cf985d2f4565281d171ff59589f7747336f1ddd NaN 169743 169743,169743 0.001 14 12 temp,hum 2020-04-14 00:00:00+00:00 15T active Max's Oyster Bar
17 ada_parms_1e2a9431da57c6df1e1512b92c440c3bab95a56e_20200514175654285970Z model_lstm 6f9003a58d4c7c5d2ca31b74738bcaf94387c005 NaN 207300 207300,207300 0.001 14 12 temp,hum 2020-04-14 00:00:00+00:00 15T active Kane's Market
18 ada_parms_7c16de08b719be5ada8a5f5243dc5869877bd468_20200514180322699476Z model_lstm 741d68518b650a65720f68a8f3407b22b4a020c3 NaN 175515 175515 0.001 14 12 temp 2020-04-14 00:00:00+00:00 15T active EMC - Elkhorn
19 ada_parms_60d957c49ee98bf1f3392b4f4b24b37e515a03e9_20200514164902709986Z model_lstm 8347e2c87f1f98eeb94e3885d161c97919d79380 NaN 175514 175514 0.001 14 12 temp 2020-04-14 00:00:00+00:00 15T active EMC Elkhorn
20 ada_parms_1473647b413dcc7b7159bd6688d48946cb21705f_20200514181559782956Z model_lstm 84486fbbacf7afc0242641d670339fd15bd63ae6 NaN 207294 207294 0.001 14 12 temp 2020-04-14 00:00:00+00:00 15T active Kane's Market
21 ada_parms_e2dd569f673fe4ac9d5e9232262665a1b55af561_20200514204112509585Z model_lstm 8b9929b0000fe13f7e68a3497b6fb5c8b276db5a NaN 207193 207193 0.001 14 12 temp 2020-04-14 00:00:00+00:00 15T active On20 Restaurant
22 ada_parms_186ab7f9feb412df95fd399f6546d48e95392ef6_20200514181328847303Z model_lstm 8e454136bebf391c9d265d9bf102ff37fc30e4e7 NaN 207296 207296 0.001 14 12 temp 2020-04-14 00:00:00+00:00 15T active Kane's Market
23 ada_parms_713ceb0988de37f4ee4ced5e94e03bf89e9e8bd0_20200514181112151954Z model_lstm 91157917dc90c6184bd87c1c6f9aac5721acc67e NaN 175514 175514,175514 0.001 14 12 temp,hum 2020-04-14 00:00:00+00:00 15T active EMC Elkhorn
24 ada_parms_9f07de91f6aebab196789f76a33781473fbef5df_20200514160003064182Z model_lstm 981922c729d5bfd3eb290a7451c98032faac4ab1 NaN 195514 195514,195514 0.001 14 12 temp,hum 2020-04-14 00:00:00+00:00 15T active Pomperaug Woods
25 ada_parms_9320a85035c6b1a46de1ae22325ee4bf816bea78_20200514165317407851Z model_lstm 98e3fe02a600b00fe7304c38453b38879ae03b58 NaN 207299 207299 0.001 14 12 temp 2020-04-14 00:00:00+00:00 15T active Kane's Market
26 ada_parms_52cd202b038fdea13e6870b3bbe862bfe889d034_20200514161100257249Z model_lstm ab0a8309ac2bf45aeb24df127c4acbd607930c95 NaN 175505 175505,175505 0.001 14 12 temp,hum 2020-04-14 00:00:00+00:00 15T active EMC Guthrie Center
27 ada_parms_cc3d8533ab9aa1bf3dee1416116695e5824bcd9f_20200514163245362629Z model_lstm b0f7c800636b349347d579e136fc90232143f7c4 NaN 175505 175505 0.001 14 12 temp 2020-04-14 00:00:00+00:00 15T active EMC Guthrie Center
28 ada_parms_1f0ede99ddea0f013065438851045db3f1ffc9b8_20200514164710297766Z model_lstm b6cffc2b9be06fe155db5531b316552675ec95a8 NaN 195517 195517,195517 0.001 14 12 temp,hum 2020-04-14 00:00:00+00:00 15T active Pomperaug Woods
29 ada_parms_5d3bc0983325ad669f1202dc68d06fcdceac17f1_20200514160622418217Z model_lstm b93bdb16b493de908152e8ce4c526359161ba88e NaN 207295 207295,207295 0.001 14 12 temp,hum 2020-04-14 00:00:00+00:00 15T active Kane's Market
30 ada_parms_9191075c57e2f62f59e2db976b9e34193fb44f76_20200514155357205330Z model_lstm c5407aac2f25474bf49d7f238a0fa00bea0f47ee NaN 195550 195550,195550 0.001 14 12 temp,hum 2020-04-14 00:00:00+00:00 15T active Holmberg Orchard
31 ada_parms_2462cd00249d288ac46fa7dd2f0620a1d40c0fa4_20200514192807189199Z model_lstm cd812ace86afa4afa1e0d6db47e9cdf02ab3cf01 NaN 169740 169740 0.001 14 12 temp 2020-04-14 00:00:00+00:00 15T active Max's Oyster Bar
32 ada_parms_062fce53d5bac51a1acd3877d513dbc8f717c505_20200514183507996224Z model_lstm d7e15d77d3243f2aadd3eb619db1367a311c5b4e NaN 195550 195550 0.001 14 12 temp 2020-04-14 00:00:00+00:00 15T active Holmberg Orchard
33 ada_parms_59fcfbaa8e1d8087862856ef8f31b5245a72aabb_20200514205956698426Z model_lstm e04b537313b7949a618b70ac67ce445c1358d32e NaN 175509 175509 0.001 14 12 temp 2020-04-14 00:00:00+00:00 15T active EMC Guthrie Center
34 ada_parms_2303800e5131d43f2099091fe30cf5a8c89f958e_20200514235118820377Z model_lstm e0e121085c0001186558bddd8f2f2326dcc1b942 NaN 207193 207193,207193 0.001 14 12 temp,hum 2020-04-14 00:00:00+00:00 15T active On20 Restaurant
35 ada_parms_651745b101f723bdf0c704457b9b860e19fd1864_20200514215815502549Z model_lstm e35965acd9cbe1fa9966ba8c9dd935effe77f5a2 NaN 179177 179177,179177 0.001 14 12 temp,hum 2020-04-14 00:00:00+00:00 15T active Max Downtown
36 ada_parms_d0840cc54eefcf29736711ce57cd9b76861d628a_20200514220751613814Z model_lstm e995912c6da003eb92d93dbe21afff48dca694ac NaN 179172 179172,179172 0.001 14 12 temp,hum 2020-04-14 00:00:00+00:00 15T active Max Downtown
37 ada_parms_c015bf90b7df49fff3baabb18c49953235f4f85e_20200514222457721588Z model_lstm ebd7c2700222dad5984e2391dd767ed47b67aabe NaN 207303 207303,207303 0.001 14 12 temp,hum 2020-04-14 00:00:00+00:00 15T active Kane's Market
38 ada_parms_3fa8025b95900b7a98e30110bd8e0b16910aae49_20200514162800909092Z model_lstm ecc784af72ea07c453b10c684eb267b650fc0224 NaN 207299 207299,207299 0.001 14 12 temp,hum 2020-04-14 00:00:00+00:00 15T active Kane's Market
39 ada_parms_16f0695b0db5662375b05d23b521a23f127d8eb2_20200514210342706190Z model_lstm ee8aa1ed12090a15f360b59b9f0a338958af0b75 NaN 207301 207301 0.001 14 12 temp 2020-04-14 00:00:00+00:00 15T active Kane's Market
40 ada_parms_1d8b1bc5d0cfa70ac6d21dd9decb2ad64c81e9cf_20200514235648244911Z model_lstm f13303738a49488aed66947aa6b333c0e3ac164c NaN 169740 169740,169740 0.001 14 12 temp,hum 2020-04-14 00:00:00+00:00 15T active Max's Oyster Bar
41 ada_parms_5be413a0a03f972a574e1bd47656055cee180e1a_20200514222716100080Z model_lstm fffd5db33793e5f7f394ce8669ad831d11461741 NaN 207192 207192 0.001 14 12 temp 2020-04-14 00:00:00+00:00 15T active On20 Restaurant
42 ada_parms_a53c7defb61105038e213c344c4c12f6625f71b9_20200514144521125831Z model_lstm 0758302fe0911623d4d6d6769344e4424f3d2185 NaN 179172 179172,179172 0.001 14 24 temp,hum 2020-04-14 00:00:00+00:00 15T active Max Downtown
43 ada_parms_92676368eb793ef6442ab0c00f0a7312aa5f07a6_20200513194039207181Z model_lstm 0fd7add4349c761c7b930f68bb829817390b8e5a NaN 179177 179177 0.001 14 24 temp 2020-04-14 00:00:00+00:00 15T active Max Downtown
44 ada_parms_85035d344f1698e1cce0889963b613de2af94d84_20200514062849962439Z model_lstm 11fa141f5abe7c133098ba6ef772730d97728dd4 NaN 207298 207298 0.001 14 24 temp 2020-04-14 00:00:00+00:00 15T active Kane's Market
45 ada_parms_5293e6a7ab0e7b4a31a0742a88372d2fd42d9547_20200514114751212093Z model_lstm 2226535cf617713ecbc2196a0bf12dc54e6b94d7 NaN 175509 175509,175509 0.001 14 24 temp,hum 2020-04-14 00:00:00+00:00 15T active EMC Guthrie Center
46 ada_parms_3cc3620d6d26d3f2141f5cc8c8fb5fe18d3222c2_20200513190703445979Z model_lstm 266b88dd4955a34dc886c9c160112385c6671d86 NaN 195517 195517,195517 0.001 14 24 temp,hum 2020-04-14 00:00:00+00:00 15T active Pomperaug Woods
47 ada_parms_bd67ae40659285020127628a69db654de2fc8fd9_20200514080238133103Z model_lstm 2d461f54fda9c142970237d5ab26442f615520bd NaN 207295 207295,207295 0.001 14 24 temp,hum 2020-04-14 00:00:00+00:00 15T active Kane's Market
48 ada_parms_e822343010f43a2b367eea0cefcf2ed3b1a14a70_20200514102747121773Z model_lstm 2dc783c92abe907d4d247693def9515607e4a7f6 NaN 175515 175515 0.001 14 24 temp 2020-04-14 00:00:00+00:00 15T active EMC - Elkhorn
49 ada_parms_247e91caddfed971050410f77ef498574ae8e023_20200513191314482138Z model_lstm 30af1e8e4cc1859fb59d33a4473c246418ec177c NaN 195514 195514 0.001 14 24 temp 2020-04-14 00:00:00+00:00 15T active Pomperaug Woods
50 ada_parms_05df5d7dda0fded21542f3d57cf4858d6e78eb85_20200513164105294293Z model_lstm 31f3bf267ddac5c010c6de565a90d9b08049c84b NaN 175509 175509 0.001 14 24 temp 2020-04-14 00:00:00+00:00 15T active EMC Guthrie Center
51 ada_parms_bddbc1b51d0e9acc2b4f1cf618f6392a58a1663d_20200513175727318676Z model_lstm 3ef5d272a812f42a2a006a515af5cb7594fb2270 NaN 207192 207192,207192 0.001 14 24 temp,hum 2020-04-14 00:00:00+00:00 15T active On20 Restaurant
52 ada_parms_4d250be8fe922dbe0ce2d76dd6f1e26450559e22_20200514061125303000Z model_lstm 49870082ef26d368567151aed149427841aa4d29 NaN 207301 207301 0.001 14 24 temp 2020-04-14 00:00:00+00:00 15T active Kane's Market
53 ada_parms_5da2ccac379b715eca3eb939893209fd074f3e1e_20200514021326418854Z model_lstm 4bff6cbd430b589c313e6d6454994d1fd610452e NaN 175505 175505,175505 0.001 14 24 temp,hum 2020-04-14 00:00:00+00:00 15T active EMC Guthrie Center
54 ada_parms_1133e85ac687705e5cb7574a4e2202fbc092a62a_20200514023958198352Z model_lstm 4c47b05f7161e39a6c8c08f3f04ba82fc29890c8 NaN 207298 207298,207298 0.001 14 24 temp,hum 2020-04-14 00:00:00+00:00 15T active Kane's Market
55 ada_parms_ec0171116844612164092ed4d54c1dd70e823b3c_20200514135440711856Z model_lstm 4cb6ca492a986c3e20412229bb7b6e0a1ebb38f5 NaN 195550 195550,195550 0.001 14 24 temp,hum 2020-04-14 00:00:00+00:00 15T active Holmberg Orchard
56 ada_parms_c7b54a370c178ae7d33451b2287753f4f0f0cf25_20200514072431310890Z model_lstm 5c1026361bde615c81198edb34e87b5b7842f163 NaN 207303 207303 0.001 14 24 temp 2020-04-14 00:00:00+00:00 15T active Kane's Market
57 ada_parms_efcdbbb45435cadc9bf801af3d3d5c8dcb874cf9_20200513152225114627Z model_lstm 68755444a4dd2ade98b7a8c589f593ed4059e534 NaN 175505 175505 0.001 14 24 temp 2020-04-14 00:00:00+00:00 15T active EMC Guthrie Center
58 ada_parms_ca1b69a8cdeb272f034413ee92a3366a10ce004e_20200513151212069976Z model_lstm 6dd8fc97f5fbd505f620aa3066a3463e3e34cbeb NaN 207192 207192 0.001 14 24 temp 2020-04-14 00:00:00+00:00 15T active On20 Restaurant
59 ada_parms_dbd3b766529ccef7be5f095af20a2b5ea0c68c74_20200514014744176219Z model_lstm 6def78ea3a384a02cb01f7e887ce942df0676ac1 NaN 207193 207193,207193 0.001 14 24 temp,hum 2020-04-14 00:00:00+00:00 15T active On20 Restaurant
60 ada_parms_855d4fec9a049984a4fdec6b3168cd4b877a559f_20200513164428044186Z model_lstm 75feaeb816d337c972fe938263275a833c02228b NaN 175514 175514 0.001 14 24 temp 2020-04-14 00:00:00+00:00 15T active EMC Elkhorn
61 ada_parms_6d1000ecd6dd0046ef234d39f441921eecb67703_20200514020529539924Z model_lstm 786a3ba103bddd5092a581d9efa8c5ff6bd6d6ff NaN 207296 207296,207296 0.001 14 24 temp,hum 2020-04-14 00:00:00+00:00 15T active Kane's Market
62 ada_parms_25a7fb3095fa4c1f9a19d7a0c5c4f47b9de68eaf_20200514065132819522Z model_lstm 87270058593104ebb27369b2571c6f87666bcad7 NaN 207299 207299 0.001 14 24 temp 2020-04-14 00:00:00+00:00 15T active Kane's Market
63 ada_parms_197acba2d69a43d4502f10affab681039b24e714_20200514142132610776Z model_lstm 8a8a38743470089b4b650a3dd5425344a9f3ca2b NaN 175515 175515,175515 0.001 14 24 temp,hum 2020-04-14 00:00:00+00:00 15T active EMC - Elkhorn
64 ada_parms_4a1ca29b67f0398bc54572ecb0df7d1d1f142873_20200514025019873622Z model_lstm 8ef4fa2de0d8eb3ca2337e4f6cb567eaabaf5593 NaN 169743 169743 0.001 14 24 temp 2020-04-14 00:00:00+00:00 15T active Max's Oyster Bar
65 ada_parms_fe8f10842f6017a440660c71f137617404ca08c7_20200513211829330406Z model_lstm 94d0554ecb2bfda45d842b17f5e2ffaf75e55f9b NaN 207303 207303,207303 0.001 14 24 temp,hum 2020-04-14 00:00:00+00:00 15T active Kane's Market
66 ada_parms_c736a6149a7eec8ef0d2d98bb1caef6e4241321e_20200514041607119442Z model_lstm 9da0a05570786aba05ae1b946cdf316e7a3dbcaf NaN 207193 207193 0.001 14 24 temp 2020-04-14 00:00:00+00:00 15T active On20 Restaurant
67 ada_parms_9f0e84b7d7f7da8880a3f12c9391d0c33226e86d_20200513150612921456Z model_lstm a3713a98c8ac036996ae8dcce36db288b4be6851 NaN 175514 175514,175514 0.001 14 24 temp,hum 2020-04-14 00:00:00+00:00 15T active EMC Elkhorn
68 ada_parms_bcea30835402f4b6d6675937e5ed70a07ec236a2_20200513194620897894Z model_lstm b41bc741e8f8858668f35855d6d87f3e43ceb989 NaN 207300 207300 0.001 14 24 temp 2020-04-14 00:00:00+00:00 15T active Kane's Market
69 ada_parms_4b4f7290b0764c2d4588d0df4ccd6be763bd7fca_20200514073629112294Z model_lstm b8944c0b61b33ef3f3c39175cfed0a004a80a5e3 NaN 207300 207300,207300 0.001 14 24 temp,hum 2020-04-14 00:00:00+00:00 15T active Kane's Market
70 ada_parms_9591d43c04cb563a98cf7af9ecfcd6a83edcd874_20200513212157711508Z model_lstm bcd8d06e2638f438f8cd1eb3830d983efc1a4aa3 NaN 195517 195517 0.001 14 24 temp 2020-04-14 00:00:00+00:00 15T active Pomperaug Woods
71 ada_parms_1a1a31998af4c30e1ecda97f2a1346768a2e0e68_20200513151453244881Z model_lstm c6ad8dc1e2beb98d26bd6b55cfede6c2d9bee938 NaN 169740 169740 0.001 14 24 temp 2020-04-14 00:00:00+00:00 15T active Max's Oyster Bar
72 ada_parms_d61a27671a9415c33da221df8679c42134589d32_20200513225410120012Z model_lstm c72309c7d728f53c67995114101154aba629a49b NaN 207294 207294,207294 0.001 14 24 temp,hum 2020-04-14 00:00:00+00:00 15T active Kane's Market
73 ada_parms_58eaf938daec7d701947df1ced397c34a9bccd75_20200514024415317951Z model_lstm c870d90e1d0e22906c6a83c65e29d9067589e9b0 NaN 207296 207296 0.001 14 24 temp 2020-04-14 00:00:00+00:00 15T active Kane's Market
74 ada_parms_c9e2b92ad4e4e185f3f4caf56080f93abd25cd4e_20200513174039125787Z model_lstm c9ba8bd7ce219226ec9d5f3dee39e36ec2c738a7 NaN 207301 207301,207301 0.001 14 24 temp,hum 2020-04-14 00:00:00+00:00 15T active Kane's Market
75 ada_parms_853f60021991bdadd3143e93587d1c0a880d3c5b_20200514022511502517Z model_lstm d2a615e544da5c4d6b4f6d0b9d635eb1095e5895 NaN 195514 195514,195514 0.001 14 24 temp,hum 2020-04-14 00:00:00+00:00 15T active Pomperaug Woods
76 ada_parms_cf2509085d66045052759b4d056c506f92120fa3_20200514085809498571Z model_lstm dcff2102d930ad04df552bb017ffcc58cccb880b NaN 195550 195550 0.001 14 24 temp 2020-04-14 00:00:00+00:00 15T active Holmberg Orchard
77 ada_parms_88e27b066260e940f8582cf19bfb8afeea761126_20200514102428108114Z model_lstm e2cc7e3a1b2b2551eedefe12316b63562d553e4a NaN 179172 179172 0.001 14 24 temp 2020-04-14 00:00:00+00:00 15T active Max Downtown
78 ada_parms_abd9dc80543d4a96f513835fadeb0cf2c100a331_20200514083129019174Z model_lstm f152205bdd6eebf84737fa2ac4fa1520b10ad1da NaN 169743 169743,169743 0.001 14 24 temp,hum 2020-04-14 00:00:00+00:00 15T active Max's Oyster Bar
79 ada_parms_e223535daa9d65453a4e41238073073f9b6eddee_20200514055737119303Z model_lstm f23f8a91e71f34ea0a79e46203b91ea2b4839eda NaN 207295 207295 0.001 14 24 temp 2020-04-14 00:00:00+00:00 15T active Kane's Market
80 ada_parms_f0124faa8a317be3f210c20f51b07a20061eb07f_20200514054628933614Z model_lstm f2cfff395512326af98e8292f107bc2260e583f3 NaN 179177 179177,179177 0.001 14 24 temp,hum 2020-04-14 00:00:00+00:00 15T active Max Downtown
81 ada_parms_bb3d2b2cca5163501822b8e080b4daba0a8bbc82_20200514115322723779Z model_lstm f8ea67b728cb59361d71bf51b60cfc0bd555940a NaN 207294 207294 0.001 14 24 temp 2020-04-14 00:00:00+00:00 15T active Kane's Market
82 ada_parms_33d1e5bb885f78edad10b9c6bf105c7b3809a3a3_20200514002627879751Z model_lstm f9eb2c42df0765d0f8ee7225974e09a24e647062 NaN 207299 207299,207299 0.001 14 24 temp,hum 2020-04-14 00:00:00+00:00 15T active Kane's Market
83 ada_parms_36e3dc98f48a722992ca68df8176280a4d72eef3_20200513171404464213Z model_lstm f9fd6c6e59817f3899d46c3d0473a99e950e8973 NaN 169740 169740,169740 0.001 14 24 temp,hum 2020-04-14 00:00:00+00:00 15T active Max's Oyster Bar
84 ada_parms_7234263c8c0a4854d20b34cb18103420091d9970_20200515082014378898Z model_lstm 2d59810c21595d403545a9ca2cc64ab2b3e99385 NaN 207300 207300,207300 0.001 14 96 temp,hum 2020-04-14 00:00:00+00:00 15T active Kane's Market
85 ada_parms_576fd0d1d7a30b308625cf079cd860ec89006265_20200515011946732855Z model_lstm 4b2a3b438586e96fc4e5587dc546b2fba8b0fbe3 NaN 169743 169743 0.001 14 96 temp 2020-04-14 00:00:00+00:00 15T active Max's Oyster Bar
86 ada_parms_0c06354d3497e7480a69a693bbb477efed79a689_20200515132429092904Z model_lstm 520276eeb74a12251103593e147ee19a8e2252a1 NaN 179177 179177,179177 0.001 14 96 temp,hum 2020-04-14 00:00:00+00:00 15T active Max Downtown
87 ada_parms_1fe1be1b630e2da41c8c2d283baf13fc73cdf793_20200515054108623269Z model_lstm 7e337336dbe8bd0275d3adec251ff62ede2c270a NaN 175509 175509 0.001 14 96 temp 2020-04-14 00:00:00+00:00 15T active EMC Guthrie Center
88 ada_parms_1929bda68bd45727ed15e0efed1ed8ed504bf5f2_20200514175630697947Z model_lstm NaN NaN 195550 195550,195550 0.001 14 96 temp,hum 2020-04-14 00:00:00+00:00 15T active Holmberg Orchard
89 ada_parms_1d390fe84090688b930c7d7b6bd62a1b9b3f3ff3_20200514175630360056Z model_lstm NaN NaN 207192 207192 0.001 14 96 temp 2020-04-14 00:00:00+00:00 15T active On20 Restaurant
90 ada_parms_1ebd3ce8b045a693653d7b1a3d0525f714ce1ce0_20200514175630247856Z model_lstm NaN NaN 207303 207303 0.001 14 96 temp 2020-04-14 00:00:00+00:00 15T active Kane's Market
91 ada_parms_1ecf13c4c08bce3b4b033bc6267a86a42e897172_20200514175629628332Z model_lstm NaN NaN 179177 179177 0.001 14 96 temp 2020-04-14 00:00:00+00:00 15T active Max Downtown
92 ada_parms_1eeb3c34c4badec48689670934416c29460414c1_20200514175628514627Z model_lstm NaN NaN 207298 207298,207298 0.001 14 96 temp,hum 2020-04-14 00:00:00+00:00 15T active Kane's Market
93 ada_parms_219b0f030e9b45090d10b8760f9ef32a1650ebd2_20200514175628949561Z model_lstm NaN NaN 195517 195517,195517 0.001 14 96 temp,hum 2020-04-14 00:00:00+00:00 15T active Pomperaug Woods
94 ada_parms_283d5c6d2d7e68baca6518ba9239c956cfe698dc_20200514175630637570Z model_lstm NaN NaN 175515 175515 0.001 14 96 temp 2020-04-14 00:00:00+00:00 15T active EMC - Elkhorn
95 ada_parms_311c80a54739e4454f27c3e920b5c87180d7db54_20200514175629458262Z model_lstm NaN NaN 195514 195514,195514 0.001 14 96 temp,hum 2020-04-14 00:00:00+00:00 15T active Pomperaug Woods
96 ada_parms_323948306cfb1ac2a82769b5c8eefad6d3fec56f_20200514175629127593Z model_lstm NaN NaN 207301 207301 0.001 14 96 temp 2020-04-14 00:00:00+00:00 15T active Kane's Market
97 ada_parms_39348a7f163947aec3c06d0372e3dde4e7d7f07c_20200514175629511996Z model_lstm NaN NaN 175514 175514 0.001 14 96 temp 2020-04-14 00:00:00+00:00 15T active EMC Elkhorn
98 ada_parms_3e63a7b8836baa7530a09c673b8240d7d48fe567_20200514175630071169Z model_lstm NaN NaN 207295 207295,207295 0.001 14 96 temp,hum 2020-04-14 00:00:00+00:00 15T active Kane's Market
99 ada_parms_4d7525c95985a5bd166725db1c65cf537c588a46_20200514175629345382Z model_lstm NaN NaN 207296 207296,207296 0.001 14 96 temp,hum 2020-04-14 00:00:00+00:00 15T active Kane's Market
100 ada_parms_55d7c5c7b5a06f69ccb2762c53fa6b8e8d066a5a_20200514175629952540Z model_lstm NaN NaN 207294 207294 0.001 14 96 temp 2020-04-14 00:00:00+00:00 15T active Kane's Market
101 ada_parms_58a5a8517843f260e21fdf9fe0bd6ca0ade1cdd2_20200514175629180068Z model_lstm NaN NaN 175509 175509,175509 0.001 14 96 temp,hum 2020-04-14 00:00:00+00:00 15T active EMC Guthrie Center
102 ada_parms_5d9983e67030641a7a83792e45012bb3de02064c_20200514175628767667Z model_lstm NaN NaN 207299 207299 0.001 14 96 temp 2020-04-14 00:00:00+00:00 15T active Kane's Market
103 ada_parms_6e4a623f00b2a2434a00a6935b11ddaec9fab2ab_20200514175630524869Z model_lstm NaN NaN 207193 207193,207193 0.001 14 96 temp,hum 2020-04-14 00:00:00+00:00 15T active On20 Restaurant
104 ada_parms_722c013b52d90bc9b5243660cf12c1e932c6f13a_20200514175630581429Z model_lstm NaN NaN 207295 207295 0.001 14 96 temp 2020-04-14 00:00:00+00:00 15T active Kane's Market
105 ada_parms_74595a67ea250e8e7bfd9f0ad990735dc582e2f0_20200514175630303330Z model_lstm NaN NaN 207301 207301,207301 0.001 14 96 temp,hum 2020-04-14 00:00:00+00:00 15T active Kane's Market
106 ada_parms_74b3dccc94fa0b9eb2c4507ad53f7d064097c76b_20200514175629572357Z model_lstm NaN NaN 179172 179172 0.001 14 96 temp 2020-04-14 00:00:00+00:00 15T active Max Downtown
107 ada_parms_751cccde4dc55ff1ad0c09f16d889dff2bbe3dbf_20200514175628593520Z model_lstm NaN NaN 175515 175515,175515 0.001 14 96 temp,hum 2020-04-14 00:00:00+00:00 15T active EMC - Elkhorn
108 ada_parms_8b173a232f184a676695e9303acad4aadc558800_20200514175629292156Z model_lstm NaN NaN 175505 175505,175505 0.001 14 96 temp,hum 2020-04-14 00:00:00+00:00 15T active EMC Guthrie Center
109 ada_parms_8dd9c58be53877941f5ec900953a9ae2ab323698_20200514175629233782Z model_lstm NaN NaN 207296 207296 0.001 14 96 temp 2020-04-14 00:00:00+00:00 15T active Kane's Market
110 ada_parms_90d17e1cfaa1cff70986943138634cebaae11cc4_20200514175630756201Z model_lstm NaN NaN 175505 175505 0.001 14 96 temp 2020-04-14 00:00:00+00:00 15T active EMC Guthrie Center
111 ada_parms_a0b1ab3aa3b24382a054fb715a3d0be2883baa3b_20200514175629402910Z model_lstm NaN NaN 207298 207298 0.001 14 96 temp 2020-04-14 00:00:00+00:00 15T active Kane's Market
112 ada_parms_a9c3f3f733816dedbfe0e91640072db17f95ec79_20200514175630134858Z model_lstm NaN NaN 207192 207192,207192 0.001 14 96 temp,hum 2020-04-14 00:00:00+00:00 15T active On20 Restaurant
113 ada_parms_af16afe84c69f42253cd9bc3c4641544c034a126_20200514175629834718Z model_lstm NaN NaN 179172 179172,179172 0.001 14 96 temp,hum 2020-04-14 00:00:00+00:00 15T active Max Downtown
114 ada_parms_b0317d16a47bc941689a18fad029f755a41b5eff_20200514175629893861Z model_lstm NaN NaN 169743 169743,169743 0.001 14 96 temp,hum 2020-04-14 00:00:00+00:00 15T active Max's Oyster Bar
115 ada_parms_badc2fc6b0124fd2f88a704497ad3fabaf1a24ef_20200514175628825401Z model_lstm NaN NaN 207299 207299,207299 0.001 14 96 temp,hum 2020-04-14 00:00:00+00:00 15T active Kane's Market
116 ada_parms_bf4c6d013591f5b1737e6cd403dd43cb7accaa91_20200514175630020231Z model_lstm NaN NaN 207300 207300 0.001 14 96 temp 2020-04-14 00:00:00+00:00 15T active Kane's Market
117 ada_parms_c229a387ba89e2f6b1da4550a8ee55e61e2f5e59_20200514175629776916Z model_lstm NaN NaN 175514 175514,175514 0.001 14 96 temp,hum 2020-04-14 00:00:00+00:00 15T active EMC Elkhorn
118 ada_parms_c3538c2db3f262841027344fe741a9364243d023_20200514175628707857Z model_lstm NaN NaN 195514 195514 0.001 14 96 temp 2020-04-14 00:00:00+00:00 15T active Pomperaug Woods
119 ada_parms_ce68ca76453c7d6ff8ee331f093cb6cfc3515325_20200514175630875570Z model_lstm NaN NaN 207303 207303,207303 0.001 14 96 temp,hum 2020-04-14 00:00:00+00:00 15T active Kane's Market
120 ada_parms_d34b2a8b271d38801a2e8919fd88f295f437168e_20200514175628880972Z model_lstm NaN NaN 169740 169740,169740 0.001 14 96 temp,hum 2020-04-14 00:00:00+00:00 15T active Max's Oyster Bar
121 ada_parms_d5a2807991db6c077756883c8f5d4633c3104887_20200514175629709051Z model_lstm NaN NaN 195517 195517 0.001 14 96 temp 2020-04-14 00:00:00+00:00 15T active Pomperaug Woods
122 ada_parms_ed95eee87a172723214c347582d029bb06eba285_20200514175630188283Z model_lstm NaN NaN 207193 207193 0.001 14 96 temp 2020-04-14 00:00:00+00:00 15T active On20 Restaurant
123 ada_parms_f192ba3f3314c2a00a07d58732e54b1a0e38cbd8_20200514175630472541Z model_lstm NaN NaN 207294 207294,207294 0.001 14 96 temp,hum 2020-04-14 00:00:00+00:00 15T active Kane's Market
124 ada_parms_f786b809f8467281ba41114c3e0fd0622ad62837_20200514175628652010Z model_lstm NaN NaN 169740 169740 0.001 14 96 temp 2020-04-14 00:00:00+00:00 15T active Max's Oyster Bar
125 ada_parms_feba889ea328ae17ac62c8a66c395da27fc57327_20200514175630414878Z model_lstm NaN NaN 195550 195550 0.001 14 96 temp 2020-04-14 00:00:00+00:00 15T active Holmberg Orchard

2. Model outputs transition

Daily maximums of anomaly output values are categorized into 3 groups ($[0,0.7),[0.7,0.9)\textrm{ and }[0.9,1.0]$) and plotted with a sankey diagram to see how they makes transitions among the groups.

In [29]:
convert_to_formated_datetime = lambda dt_str: pd.Timestamp(dt_str).strftime('%Y-%m-%dT%H:%M:%SZ')
qry = '''{
    "from": {
        "parametrizeddatasets": {
            "name": "adm_anomaly_outputs_dataset",
            "parameters": [
                {"t1": "%s"},
                {"t2": "%s"}
            ]
        }
    },
    "includeArchivedData": False,
    "select": [
        { "count": "x" },
        { "min": "x" },
        { "avg": "x" },
        { "max": "x" },
        { "stddeviation": "x" }
    ],
    "groupByTime": {
        "field": "x_timestamp",
        "interval": "day", # "hour"
     #   "timeZone": "America/New_York"
    },
    "groupBy": [
        "x_object.x_device_id"
    ],
    "limitLastGroup": 100000
}'''
In [30]:
res_lst = [
    ms.run_mnubo_query(eval(qry % (convert_to_formated_datetime(x), convert_to_formated_datetime(x+pd.Timedelta('1D')))))
    for x in pd.date_range('2020-04-14','2020-05-16')
]
res = pd.concat(res_lst)
In [33]:
#nodes, state_transition = gen_sankey_tsdata(res, 'hour', 'x_object.x_device_id', 'MAX(x)', ypos=[.7, 0.2, 0.1])
nodes, state_transition = gen_sankey_tsdata(res, 'day', 'x_object.x_device_id', 'MAX(x)', ypos=[.7, 0.2, 0.1], value_cuts=[.0, 0.7, 0.9, 1.0])
In [34]:
import plotly.graph_objects as go
fig = go.Figure(data=[go.Sankey(
    valueformat = ".0f",
    valuesuffix = "#",    
    arrangement = "freeform",
    node = dict(
        pad = 40,
        thickness = 3,
        line = dict(color = "black", width = 0.1),
        label=nodes[['node_t','state_t']].apply(lambda x: x.astype(str), axis=0).apply(lambda x: '_'.join(x), axis=1).values,        
        color = nodes.state_color.values,
        x = nodes.xpos.values,
        y = nodes.ypos.values
    ),
    link = dict(
        source = state_transition._nid1,
        target = state_transition._nid2,
        value = state_transition.cnt,
        label = state_transition['x_object.x_device_id'],
        color = state_transition.link_color
    
  ))])

fig.update_layout(title_text="Anomaly Sankey Diagram", font_size=10, width=2200, height=1000)
fig.show()

3. Anomalies detected

Sensor data and their computed anomaly values for sensors of the third category (anomaly value $\ge$ 0.9)

Anomaly values are computed from 2020-04-19 and sensor data of 2 weeks before the time were used to train models.

Colors in upper graphs represent anomaly values from models of different parameters with the variations mentions in the section 1.

Colors in lower graphs represent different channels of corresponding sensors.

In [35]:
params_state_red = state_transition[state_transition._s2==state_transition._s2.cat.categories[-1]]['x_object.x_device_id'].unique()
In [60]:
ap = AnomalyPlotter()
t1, t2 = '2020-03-30', '2020-05-15'
plots = ap.get_psets_plots(psets[psets.x_device_id.isin(params_state_red)], t1, t2, acc_sensor_info)

175514
175515
179177
207295
207299
In [72]:
for p in plots:
    print(p + p9.theme(figure_size=(15,10)) + p9.xlab(''))

<ggplot: (-9223363247827416597)>

<ggplot: (8789026689961)>

<ggplot: (8789026485957)>

<ggplot: (8789026469952)>

<ggplot: (-9223363247828547496)>